package com.xuyh.pdf.utils.ttjspider;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.date.DateTime;
import cn.hutool.core.date.DateUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.IdUtil;
import cn.hutool.core.util.NumberUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.crypto.SecureUtil;
import cn.hutool.http.HttpUtil;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.Page;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;

/**
 * @author xuyh0817
 * @version 1.0.0
 * @email xuyh0817@foxmail.com
 * @date 2024/9/23 12:32
 * @doc
 * @website
 * @description 抓取数据
 */
@Slf4j
public class SpiderArticle {
    /**
     * 主地址
     */
    private String baseUrl;

    /**
     * 存储文件路径
     */
    private String storageFileRoot = "C:\\xuyh\\docker\\nginx\\html\\cmsfile";
    /**
     * 临时空间
     */
    private String storageFileTmpRoot = "C:\\xuyh\\docker\\nginx\\html\\cmsfile\\tmp\\";
    /**
     * 访问文件路径
     */
    private String visitFileRoot = "/cmsfile";

    /**
     * 分页总数
     */
    private Integer pageTotal = 1;

    /**
     * 分页模板
     */
    private String pageUrlTmp = "index{}.html";


    private Integer timeout = 10000;

    public SpiderArticle(String baseUrl, Integer pageTotal) {
        this.baseUrl = baseUrl;
        this.pageTotal = pageTotal;
    }

    public List<Article> spiderHtml() throws Exception {
        List<Article> result = CollUtil.newArrayList();

        // 开始遍历获取每个文章
        for (int i = 0; i < pageTotal; i++) {
            String pageUrl = StrUtil.format(pageUrlTmp, i == 0 ? StrUtil.EMPTY : "_" + i);
            result.addAll(this.doParseArticleList(this.baseUrl + pageUrl));
        }
        return result;
    }

    private String doSpiderHtml(String url) throws Exception {
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        //htmlunit 对css和javascript的支持不好，所以请关闭之
        webClient.getOptions().setJavaScriptEnabled(true);
        webClient.getOptions().setCssEnabled(false);
        Page page = webClient.getPage(url);
        webClient.waitForBackgroundJavaScript(timeout);
        String html = null;
        if (page.isHtmlPage()) {
            HtmlPage htmlPage = (HtmlPage) page;
            html = htmlPage.asXml();
        }
        webClient.close();

        return html;
    }

    private List<Article> doParseArticleList(String pageUrl) throws Exception {
        List<Article> result = CollUtil.newArrayList();
        String html = this.doSpiderHtml(pageUrl);
        if (StrUtil.isBlank(html)) {
            return new ArrayList<>();
        }
        Document document = Jsoup.parse(html);

        Elements elements = document.select("div.dz-gc2r ul.newsul li");
        for (Element element : elements) {
            Element a = element.selectFirst("a");
            Element span = element.selectFirst("span");
            Article article = new Article();
            article.setTitle(StrUtil.trim(a.text()));
            article.setCreateTime(DateUtil.parse(StrUtil.trim(span.text()), "yyyy-MM-dd"));
            result.add(article);
            String href = a.attr("href");
            href = StrUtil.sub(href, 2, href.length());
            doParseArticle(article, href);
        }
        return result;
    }

    private void doParseArticle(Article article, String url) throws Exception {
        String html = this.doSpiderHtml(this.baseUrl + url);
        if (StrUtil.isBlank(html)) {
            return;
        }
        Document document = Jsoup.parse(html);

        doParseArticle1(article, document, StrUtil.split(url, StrUtil.SLASH).get(0) + StrUtil.SLASH);
    }

    private void doParseArticle1(Article article, Document document, String url) {
        // 先查找 z4-b-ww
        Elements elements = document.select("div.dz-xc3con");
        if (elements.isEmpty()) {
            return;
        }
        Element parent = this.getParent(document);
        Element trsEditor = this.getTRSEditor(document);
        String html;
        if (trsEditor == null && parent != null) {
            this.removeScriptAndStyle(parent);
            List<Attachment> attachments = this.replaceImageSrc(parent, url);
            article.setImages(attachments);
            html = parent.html();
        } else if (trsEditor != null) {
            this.removeScriptAndStyle(trsEditor);
            List<Attachment> attachments = this.replaceImageSrc(trsEditor, url);
            article.setImages(attachments);
            html = trsEditor.html();
        } else {
            return;
        }


        List<Attachment> attachments = this.getAttachment(parent, url);
        article.setAttachments(attachments);
        article.setContent(html);
        Elements spans = document.select("div.dz-xc2l span");
        for (Element span : spans) {
            if (StrUtil.contains(span.text(), "来源")) {
                String source = span.nextElementSibling().text();
                article.setSource(source);
            }
            if (StrUtil.contains(span.text(), "作者")) {
                String author = span.nextElementSibling().text();
                String source = article.getSource();
                article.setSource((StrUtil.isNotBlank(source) ? source : StrUtil.EMPTY) + (StrUtil.isNotBlank(author) ? StrUtil.SPACE + author : StrUtil.EMPTY));
            }

            if (StrUtil.contains(span.text(), "浏览量")) {
                String count = span.nextElementSibling().text();
                if (StrUtil.isNotBlank(count) && NumberUtil.isNumber(count)) {
                    article.setViewCount(NumberUtil.parseInt(count));
                } else {
                    article.setViewCount(0);
                }
            }

        }
    }


    private String replaceDiv2p(String html) {
        return html;
    }

    // 内容父级
    private Element getParent(Document document) {
        Elements select = document.select("div.dz-xc3con");
        if (select.isEmpty()) {
            return null;
        }
        return select.get(0);
    }

    // 内容实际数据
    private Element getTRSEditor(Element document) {
        Elements select = document.select("div.TRS_Editor");
        if (select.isEmpty()) {
            return null;
        }
        Element last = select.last();
        if (last.children().size() == 1) {
            return last.children().last();
        }
        return last;
    }

    // 删除script和style
    private void removeScriptAndStyle(Element parent) {
        // 删除scripts
        Elements scripts = parent.select("script");
        for (Element script : scripts) {
            script.remove();
        }
        Elements styles = parent.select("style");
        for (Element style : styles) {
            style.remove();
        }
    }

    private List<Attachment> replaceImageSrc(Element parent, String articlePrefixUrl) {
        Elements imgs = parent.select("img");
        List<Attachment> result = new ArrayList<>();
        for (Element img : imgs) {
            Attachment attachment = new Attachment();
            String src = img.attr("src");
            src = StrUtil.sub(src, 2, src.length());
            File file = new File(storageFileTmpRoot + IdUtil.fastSimpleUUID());
            HttpUtil.downloadFile(this.baseUrl + articlePrefixUrl + src, file);
            byte[] bytes = FileUtil.readBytes(file);

            ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes);
            String code = SecureUtil.md5(inputStream);
            // 将originalFilename替换成code，保留文件类型后缀
            String fileName = code + StrUtil.DOT + StrUtil.subAfter("lite.jpg", StrUtil.DOT, true);
            String filePath = storageFile(this.storageFileRoot, file, fileName);

            attachment.setFilePath(StrUtil.replace(filePath, this.storageFileRoot, StrUtil.EMPTY));
            //  访问文件
            String visitPath = StrUtil.replace(filePath, this.storageFileRoot, this.visitFileRoot);
            visitPath = StrUtil.replace(visitPath, "\\", StrUtil.SLASH);
            attachment.setVisitPath(visitPath);
            // 设置图片的地址
            img.attr("src", visitPath);

            attachment.setCode(code);
            attachment.setFileSize(bytes.length);
            attachment.setFileName(fileName);
            attachment.setFileType("image/jpeg");
            attachment.setType(Attachment.TYPE_EDITOR);
            result.add(attachment);
        }
        return result;
    }


    private List<Attachment> getAttachment(Element parent, String articlePrefixUrl) {
        Elements ps = parent.select("p");
        List<Attachment> result = new ArrayList<>();
        Element aparent = null;
        for (Element p : ps) {
            if (StrUtil.contains(p.text(), "附件下载")) {
                aparent = p.nextElementSibling();
            }
        }

        if (aparent != null) {
            Elements as = aparent.select("a");
            for (Element a : as) {

                String href = a.attr("href");
                href = StrUtil.sub(href, 1, href.length());

                File file = new File(storageFileTmpRoot + IdUtil.fastSimpleUUID());
                HttpUtil.downloadFile(this.baseUrl + articlePrefixUrl + href, file);
                byte[] bytes = FileUtil.readBytes(file);

                ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes);
                Attachment attachment = new Attachment();
                String code = SecureUtil.md5(inputStream);
                // 将originalFilename替换成code，保留文件类型后缀
                String fileName = code + StrUtil.DOT + StrUtil.subAfter(a.text(), StrUtil.DOT, true);
                String filePath = storageFile(this.storageFileRoot, file, fileName);

                attachment.setFilePath(StrUtil.replace(filePath, this.storageFileRoot, StrUtil.EMPTY));
                //  访问文件
                String visitPath = StrUtil.replace(filePath, this.storageFileRoot, this.visitFileRoot);
                visitPath = StrUtil.replace(visitPath, "\\", StrUtil.SLASH);
                attachment.setVisitPath(visitPath);
                attachment.setCode(code);
                attachment.setFileSize(bytes.length);
                attachment.setFileName(a.text());
                attachment.setFileType(FileUtil.getMimeType(file.getAbsolutePath()));
                attachment.setType(Attachment.TYPE_ARTICLE);
                result.add(attachment);
            }
        }
        return result;
    }

    /**
     * 存储文件
     *
     * @param fileStoragePath 文件存储路径
     * @param file            数据流
     * @param fileName        文件名
     * @return 文件路径
     */
    public static String storageFile(String fileStoragePath, File file, String fileName) {
        DateTime date = DateUtil.date();
        String filePath = fileStoragePath + File.separator + DateUtil.format(date, "yyyy") + File.separator + DateUtil.format(date, "MM") + File.separator + DateUtil.format(date, "dd") + File.separator;
        // 校验路径filePath是否存在，不存在则逐层创建
        FileUtil.mkdir(filePath);
        // 保存文件
        String fullFilePath = filePath + fileName;
        FileUtil.move(file, new File(fullFilePath), true);
        return fullFilePath;
    }
}
